package main

import (
	"fmt"
	"github.com/PuerkitoBio/goquery"
	"log"
	"net/http"
	"os"
	"strconv"
)

const (
	baseUrl = "https://studygolang.com/topics?p="
)

func getPageCount() int {
	return 5
}

func getResponse(url string) *http.Response {
	client := &http.Client{}
	request, _ := http.NewRequest("GET", url, nil)
	response, _ := client.Do(request)
	return response
}

func main() {

	getCount := 0
	for i := 1; i < getPageCount(); i++ {
		url := baseUrl + strconv.Itoa(i)
		response := getResponse(url)
		if response.StatusCode == 403 {
			fmt.Println("IP 已经被禁止访问")
			os.Exit(1)
		}
		if response.StatusCode == 200 {
			dom, err := goquery.NewDocumentFromResponse(response)
			if err != nil {
				log.Fatalf("失败原因 %d", response.StatusCode)
			}
			dom.Find(".topics .topic").Each(func(i int, content *goquery.Selection) {
				title := content.Find("title a").Text()
				fmt.Println(title)
			})
			getCount++
		}
	}

	fmt.Println("-------------------------------------")
	fmt.Printf("数据已经全部拉取完成，共计 %d 条\n", getCount)
}
